package com.muki.simple.core;

import java.util.ArrayList;
import java.util.List;

import com.muki.simple.configuration.InterestConf;
import com.muki.simple.entity.Content;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;


public class SpiderProcessor implements PageProcessor {

	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
	private InterestConf properties;

	public SpiderProcessor(InterestConf properties) {
		this.properties = properties;
	}

	private boolean matchLinkRoute(Page page) {
		//正则检测
		String[] dependsRegx = properties.getMatchRegx();
		if(dependsRegx!=null)
			for(String depend:dependsRegx) {
				if(page.getUrl().regex(depend).match()) {
					return true;
				}
			}

		String[] dependsXpath = properties.getMatchXpath();
		if(dependsXpath!=null)
			for(String depend:dependsXpath) {
				if(page.getUrl().xpath(depend).match()) {
					return true;
				}
			}
		return false;
	}

	private List<String> LinksSelector(Page page) {
		//正则检测
		List<String> results = new ArrayList<>();
		String[] dependsRegx = properties.getLinkRegx();
		if(dependsRegx!=null)
			for(String depend:dependsRegx) {
				results.addAll(page.getHtml().links().regex(depend).all());
			}

		String[] dependsXpath = properties.getLinkXpath();
		if(dependsXpath!=null)
			for(String depend:dependsXpath) {
				results.addAll(page.getHtml().links().xpath(depend).all());
			}
		return results;
	}

	private String contentSelector(Page page) {
		String dependsRegx = properties.getContextRegx();
		if(dependsRegx!=null && !dependsRegx.isEmpty()) {
			return page.getHtml().regex(dependsRegx).get();
		}
		String dependsXpath = properties.getContentXpath();
		if(dependsXpath!=null && !dependsXpath.isEmpty()) {
			return page.getHtml().xpath(dependsXpath).get();
		}
		return null;
	}

	private String titleSelector(Page page) {
		String dependsRegx = properties.getTitleRegx();
		if(dependsRegx!=null && !dependsRegx.isEmpty()) {
			return page.getHtml().regex(dependsRegx).get();
		}
		String dependsXpath = properties.getTitleXpath();
		if(dependsXpath!=null && !dependsXpath.isEmpty()) {
			return page.getHtml().xpath(dependsXpath).get();
		}
		return null;
	}

	private String dateSelector(Page page) {
		String dependsRegx = properties.getDateRegx();
		if(dependsRegx!=null && !dependsRegx.isEmpty()) {
			return page.getHtml().regex(dependsRegx).get();
		}
		String dependsXpath = properties.getDateXpath();
		if(dependsXpath!=null && !dependsXpath.isEmpty()) {
			return page.getHtml().xpath(dependsXpath).get();
		}
		return null;
	}

	@Override
	// process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
	public void process(Page page) {
		// 部分二：定义如何抽取页面信息，并保存下来
		if(matchLinkRoute(page)) {
			String context = contentSelector(page);
			if(context==null) {
				return;
			}
			String title = titleSelector(page);
			if(title==null)
				return;
			String datetime = dateSelector(page);
			if(datetime==null)
				return;

			Content c = new Content();
			c.setContent(context);
			c.setContent_time(datetime);
			c.setTitle(title);
			c.setLink(page.getUrl().toString());
			page.putField("content", c);
		} else {
			List<String> urls = LinksSelector(page);
			page.addTargetRequests(urls);
		}
	}

	@Override
	public Site getSite() {
		return site;
	}

}
